
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.Text;

using System.Net;
using HtmlAgilityPack;

namespace EpubLibrary
{
    public class EDSWebClient : WebClient
    {
        //time in milliseconds
        private int timeout;
        public int Timeout
        {
            get
            {
                return timeout;
            }
            set
            {
                timeout = value;
            }
        }

        public EDSWebClient()
        {
            this.timeout = 60000;
        }

        public EDSWebClient(int timeout)
        {
            this.timeout = timeout;
        }

        protected override WebRequest GetWebRequest(Uri address)
        {
            WebRequest result = base.GetWebRequest(address);
            result.Timeout = this.timeout;
            return result;
        }
    }

    public class Content
    {
        #region "Core Library"

        /// <summary>
        /// Load web data from Website
        /// </summary>
        /// <param name="url">Url of website</param>
        /// <returns>HtmlDocument content</returns>
        public static HtmlAgilityPack.HtmlDocument LoadContent(string url)
        {
            try
            {
                //WebClient web = new WebClient();
                EDSWebClient web = new EDSWebClient(40000);
                web.Encoding = Encoding.UTF8;

                //Added by manhlq, 13/06/2009
                web.Headers.Add("user-agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10)");

                string content_page = web.DownloadString(url);

                content_page = content_page.Replace("\n", "");
                content_page = content_page.Replace("\r", " ");
                content_page = content_page.Replace("\t", "");

                //Xu ly
                HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
                doc.LoadHtml(content_page);

                doc.OptionFixNestedTags = true;
                doc.OptionAutoCloseOnEnd = true;
                doc.OptionExtractErrorSourceText = true;
                doc.OptionDefaultStreamEncoding = Encoding.Unicode;
                doc.OptionWriteEmptyNodes = true;

                return doc;
            }
            catch
            {
                return null;
            }
        }

        /// <summary>
        /// Load web data from Website, charset ISO 8859
        /// </summary>
        /// <param name="url">Url of website</param>
        /// <returns>HtmlDocument content</returns>
        public static HtmlAgilityPack.HtmlDocument LoadContentISO8859(string url)
        {
            try
            {
                //Load data
                //WebClient web = new WebClient();
                EDSWebClient web = new EDSWebClient(20000);

                //Added by manhlq, 13/06/2009
                web.Headers.Add("user-agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10)");

                //web.Headers.Add("", "");
                web.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
                //web.Headers.Add("Accept-Encoding", "gzip,deflate");

                string content_page = web.DownloadString(url);

                content_page = content_page.Replace("\n", "");
                content_page = content_page.Replace("\r", "");
                content_page = content_page.Replace("\t", "");

                content_page = EDS.Common.Utilities.ConvertISO8859ToUnicode(content_page);

                //Xu ly
                HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
                doc.LoadHtml(content_page);

                doc.OptionFixNestedTags = true;
                doc.OptionAutoCloseOnEnd = true;
                doc.OptionExtractErrorSourceText = true;
                doc.OptionDefaultStreamEncoding = Encoding.Unicode;
                doc.OptionWriteEmptyNodes = true;

                return doc;
            }
            catch
            {
                return null;
            }
        }

        public static HtmlAgilityPack.HtmlDocument LoadContentGoogleBot(string url)
        {
            try
            {
                EDSWebClient web = new EDSWebClient();
                web.Encoding = Encoding.UTF8;

                //Added by manhlq, 13/06/2009
                web.Headers["User-Agent"] = "Googlebot/2.1 (+http://www.googlebot.com/bot.html";
                web.Headers["Accept-Encoding"] = "";

                string content_page = web.DownloadString(url);

                content_page = content_page.Replace("\n", "");
                content_page = content_page.Replace("\r", " ");
                content_page = content_page.Replace("\t", "");

                //Xu ly
                HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
                doc.LoadHtml(content_page);

                doc.OptionFixNestedTags = true;
                doc.OptionAutoCloseOnEnd = true;
                doc.OptionExtractErrorSourceText = true;
                doc.OptionDefaultStreamEncoding = Encoding.Unicode;
                doc.OptionWriteEmptyNodes = true;

                return doc;
            }
            catch
            {
                return null;
            }
        }

        public static HtmlAgilityPack.HtmlDocument LoadContentGoogleBotISO8859(string url)
        {
            try
            {
                EDSWebClient web = new EDSWebClient();

                //Added by manhlq, 13/06/2009
                web.Headers["User-Agent"] = "Googlebot/2.1 (+http://www.googlebot.com/bot.html";
                web.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
                web.Headers["Accept-Encoding"] = "";

                string content_page = web.DownloadString(url);

                content_page = content_page.Replace("\n", "");
                content_page = content_page.Replace("\r", " ");
                content_page = content_page.Replace("\t", "");

                //Xu ly
                HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
                doc.LoadHtml(content_page);

                doc.OptionFixNestedTags = true;
                doc.OptionAutoCloseOnEnd = true;
                doc.OptionExtractErrorSourceText = true;
                doc.OptionDefaultStreamEncoding = Encoding.Unicode;
                doc.OptionWriteEmptyNodes = true;

                return doc;
            }
            catch
            {
                return null;
            }
        }

        #endregion

        #region "Format HTMl"

        /// <summary>
        /// Removes all FONT and SPAN tags, and all Class and Style attributes.
        /// Designed to get rid of non-standard Microsoft Word HTML tags.
        /// </summary>
        public static string CleanHtml(string html)
        {
            // start by completely removing all unwanted tags 
            html = Regex.Replace(html, @"<[/]?(font|span|xml|del|ins|[ovwxp]:\w+)[^>]*?>", "", RegexOptions.IgnoreCase);
            // then run another pass over the html (twice), removing unwanted attributes 
            html = Regex.Replace(html, @"<([^>]*)(?:class|lang|style|size|face|[ovwxp]:\w+)=(?:'[^']*'|""[^""]*""|[^\s>]+)([^>]*)>", "<$1$2>", RegexOptions.IgnoreCase);
            html = Regex.Replace(html, @"<([^>]*)(?:class|lang|style|size|face|[ovwxp]:\w+)=(?:'[^']*'|""[^""]*""|[^\s>]+)([^>]*)>", "<$1$2>", RegexOptions.IgnoreCase);
            return html;
        }

        /// <summary>
        /// Remove tag html in text
        /// </summary>
        /// <param name="htmlString">string contain html tag</param>
        /// <returns>string removed while space</returns>
        public static string StripHTML(string htmlString)
        {
            string pattern = @"<(.|\n)*?>";
            return Regex.Replace(htmlString, pattern, string.Empty);
        }

        /// <summary>
        /// Remove tags html, exclute content
        /// </summary>
        /// <param name="tags">Tags html remove, ex: StripTagsHTML("span|div", html)</param>
        /// <param name="html">Input content</param>
        /// <returns>Output content</returns>
        public static string StripTagsHTML(string tags, string html)
        {
            string pattern = @"</*(" + tags + ")[^>]*>";
            return Regex.Replace(html, pattern, string.Empty);
        }

        /// <summary>
        /// Remove attributes of tag
        /// </summary>
        /// <param name="attributes">attributes</param>
        /// <param name="html">Input content</param>
        /// <returns>Output content</returns>
        public static string StripAttributesTags(string attributes, string html)
        {
            //Replace width and height of tag HTML
            //html = Regex.Replace(html, @"(" + attributes + @")\s*=\s*[""']?([^'"" >]+?)[ '""][^>]*?", String.Empty);
            html = Regex.Replace(html, "(" + attributes + "+)=(\"[^<>\"]*\"|'[^<>']*'|" + attributes + "+)", "", RegexOptions.IgnoreCase);
            return html;
        }

        /// <summary>
        /// Remove region tag javascript
        /// </summary>
        /// <param name="html">Input content</param>
        /// <returns></returns>
        public static string StripScriptTags(string html)
        {
            html = Regex.Replace(html, @"\<script(.*?)\>(.*?)\<\/script(.*?)\>", "", RegexOptions.Singleline | RegexOptions.IgnoreCase);
            return html;
        }

        private string removeSpecialAttributes(string Html)
        {
            Html = Regex.Replace(Html, @"(text-align:\s*justify(;)?)", String.Empty, RegexOptions.IgnoreCase);
            //Html = Regex.Replace(Html, @"(font-size:\s*small(;)?)", String.Empty, RegexOptions.IgnoreCase);
            Html = Regex.Replace(Html, "(font-size:\\s*[^\"](;)?)", String.Empty, RegexOptions.IgnoreCase);
            Html = Regex.Replace(Html, @"(font-style:\s*italic(;)?)", String.Empty, RegexOptions.IgnoreCase);
            Html = Html.Replace("align=\"justify\"", String.Empty);
            Html = Html.Replace("align=\"right\"", String.Empty);
            return Html;
        }  
        public static string removeInvalidTag(string invalidTag, string Html) // remote tag only, not remote tag content
        {
            //string[] tags = invalidTag.Split('|');
            //foreach (string tag in tags)
            //{
            //    if (tag != "")
            //        Html = Regex.Replace(Html, String.Format("</*{0}.*?>", tag), String.Empty, RegexOptions.Singleline | RegexOptions.IgnoreCase);
            //}

            Html = Regex.Replace(Html, String.Format("</*{0}.*?>", invalidTag), String.Empty, RegexOptions.Singleline | RegexOptions.IgnoreCase);
            return Html;
        }
        public static string removeInvalidTagAndContent(string invalidTag, string Html) // remote tag and tag content
        {
            string[] tags = invalidTag.Split('|');
            foreach (string tag in tags)
            {
                if (tag != "")
                {
                    if(tag == "input")
                        Html = Regex.Replace(Html, String.Format("<input[^>]+>", tag), String.Empty, RegexOptions.Singleline | RegexOptions.IgnoreCase);
                    Html = Regex.Replace(Html, String.Format(@"<({0})[^>]*>(.*?)</\1>", tag), String.Empty, RegexOptions.Singleline | RegexOptions.IgnoreCase);
                }
            }
            return Html;
        }
        public static string removeAllAttributes(string Html)
        {
            return Regex.Replace(Html, "<([A-Z][A-Z0-9]*)[^>]*>", "<$1>", RegexOptions.IgnoreCase);              
        }

        //public static string fixFullImagePath(string baseURL, string html)
        //{ 
            
        //}
        /// <summary>
        /// Remove Comments
        /// </summary>
        /// <param name="html">Input content</param>
        /// <returns>Output content</returns>
        public static string StripCommentsTag(string html)
        {
            html = Regex.Replace(html, "<!--.*?-->", String.Empty, RegexOptions.Compiled | RegexOptions.Singleline );
            return html;
        }


        /// <summary>
        /// Remove region tag style
        /// </summary>
        /// <param name="html">Input content</param>
        /// <returns>Output content</returns>
        public static string StripStyleTags(string html)
        {
            html = Regex.Replace(html, @"\<style(.*?)\>(.*?)\<\/style(.*?)\>", "", RegexOptions.Singleline | RegexOptions.IgnoreCase);            
            return html;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="str"></param>
        /// <returns></returns>
        public static string RemoveSpecialCharacters(string input)
        {
            return Regex.Replace(input, "[^a-zA-Z0-9_.]+", "", RegexOptions.Compiled);
        }

        public static string URLClearn(string input)
        {
            input = input.ToLower();

            //Clearn "https" string
            if (input.Contains("https"))
            {
                input = input.Replace("https", "");
            }

            //Clearn "http" string
            if (input.Contains("http"))
            {
                input = input.Replace("http", "");
            }

            //Clearn "www" string
            if (input.Contains("www"))
            {
                input = input.Replace("www", "");
            }

            //Clearn Special Characters
            input = Regex.Replace(input, "[^a-zA-Z0-9_.]+", "", RegexOptions.Compiled);

            return input;
        }


        #endregion
    }
}


